{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## 准备"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "import pandas as pd\r\n",
    "import numpy as np\r\n",
    "from lxml.html import fromstring\r\n",
    "import time\r\n",
    "from random import random\r\n",
    "import selenium\r\n",
    "import xlwt\r\n",
    "import hashlib\r\n",
    "import base64\r\n",
    "import json\r\n",
    "from urllib import parse\r\n",
    "import requests\r\n",
    "from PIL import Image\r\n",
    "from time import sleep\r\n",
    "from selenium.webdriver.common.action_chains import ActionChains"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 设置浏览器"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "# coding=utf-8\r\n",
    "from selenium import webdriver\r\n",
    "import time\r\n",
    "\r\n",
    "wd = webdriver.Chrome()\r\n",
    "wd.get(\"https://www.baidu.com\")    # 打开百度浏览器\r\n",
    "wd.find_element_by_id(\"kw\").send_keys(\"selenium\")   # 定位输入框并输入关键字\r\n",
    "wd.find_element_by_id(\"su\").click()   #点击[百度一下]搜索\r\n",
    "time.sleep(3)   #等待3秒\r\n",
    "wd.quit()   #关闭浏览器"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "\r\n",
    "from selenium import webdriver\r\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\r\n",
    "\r\n",
    "\r\n",
    "#caps=dict()\r\n",
    "#caps[\"pageLoadStrategy\"] = \"none\"   # Do not wait for full page load\r\n",
    "\r\n",
    "opts = webdriver.ChromeOptions()\r\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\r\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\r\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\r\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\r\n",
    "#opts.add_argument('blink-settings=imagesEnabled=false') #不加载图片, 提升速度\r\n",
    "#opts.add_argument('--headless') #浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败\r\n",
    "# opts.binary_location = \"C:\\portable\\PortableApps\\IronPortable\\App\\Iron\\chrome.exe\"\r\n",
    "#opts.binary_location = \"/Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --version\" #\"H:\\_coding_\\Gitee\\InternetNewMedia\\CapstonePrj2016\\chromedriver.exe\"  \r\n",
    "\r\n",
    "# \"H:\\_coding_\\Gitee\\InternetNewMedia\\CapstonePrj2016\\chromedriver.exe\"  \r\n",
    "driver = webdriver.Chrome( chrome_options = opts) #desired_capabilities=caps,"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "C:\\Users\\31435\\AppData\\Local\\Temp/ipykernel_13976/269732701.py:19: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = webdriver.Chrome( chrome_options = opts) #desired_capabilities=caps,\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 登陆"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "driver.get(\"https://www.cnki.net/\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 检查登录状态"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "#登录\r\n",
    "driver.find_element_by_id('Ecp_loginShowName1').get_attribute('innerHTML')"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'中国地质大...'"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 进入高级检索"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "#高级检索\r\n",
    "element = driver.find_element_by_id('highSearch')\r\n",
    "print(element.get_attribute('innerHTML'))\r\n",
    "element.click()"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "高级检索\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 检查窗口位置信息\n",
    "只要是多个窗口，要检查窗口信息"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "##当前窗口信息\r\n",
    "driver.current_window_handle"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'CDwindow-C72122091EEAA8579BDDDE23D6BFF8D3'"
      ]
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "##所有窗口信息\r\n",
    "driver.window_handles"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['CDwindow-C72122091EEAA8579BDDDE23D6BFF8D3',\n",
       " 'CDwindow-D2443A647695DE13B1B506236BC6BCFE']"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "#切换窗口\r\n",
    "driver.switch_to_window(driver.window_handles[1])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "C:\\Users\\31435\\AppData\\Local\\Temp/ipykernel_13976/1880812602.py:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 点击学术期刊"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "element = driver.find_element_by_xpath('//ul[@class=\"doctype-menus keji\"]/li[@data-id=\"xsqk\"]/a')\r\n",
    "print(element.get_attribute('innerHTML'))\r\n",
    "element.click()"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "<span>学术期刊</span><em></em>\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 点击专业检索"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "source": [
    "##点击专业检索\r\n",
    "element = driver.find_element_by_name('majorSearch')\r\n",
    "print(element.get_attribute('innerHTML'))\r\n",
    "element.click()"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "专业检索\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 选择期刊来源"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "#点击\"SCI\"来源\r\n",
    "element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[2]/input')\r\n",
    "element.click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "#点击\"EI\"来源\r\n",
    "element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[3]/input')\r\n",
    "element.click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "#点击\"CSSCI\"\r\n",
    "element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[5]/input')\r\n",
    "element.click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 设置搜索的query"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "source": [
    "query= 'SU = \"互联网\" '"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "source": [
    "element = driver.find_element_by_xpath('//textarea')\r\n",
    "element.clear()\r\n",
    "element.send_keys(query)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 检查检索信息"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "source": [
    "element = driver.find_element_by_xpath('//input[@value=\"检索\"]')\r\n",
    "element.get_attribute('innerHTML')\r\n",
    "element.click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "source": [
    "## 检索数量\r\n",
    "driver.find_element_by_xpath('//i[@class=\"icon icon-sort\"]').click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "source": [
    "## 设置五十条\r\n",
    "## 该处执行出现错误，需要修改（https://blog.csdn.net/songlh1234/article/details/100983248）\r\n",
    "time.sleep(1)\r\n",
    "driver.find_element_by_xpath('//*[@id=\"perPageDiv\"]/ul/li[3]/a').click()"
   ],
   "outputs": [
    {
     "output_type": "error",
     "ename": "ElementNotInteractableException",
     "evalue": "Message: element not interactable\n  (Session info: chrome=94.0.4606.71)\n",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mElementNotInteractableException\u001b[0m           Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_13976/4211886273.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m## 设置五十条\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_xpath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'//*[@id=\"perPageDiv\"]/ul/li[3]/a'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclick\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mD:\\Python39\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36mclick\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     78\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclick\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     79\u001b[0m         \u001b[1;34m\"\"\"Clicks the element.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 80\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_execute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCLICK_ELEMENT\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     81\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0msubmit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Python39\\lib\\site-packages\\selenium\\webdriver\\remote\\webelement.py\u001b[0m in \u001b[0;36m_execute\u001b[1;34m(self, command, params)\u001b[0m\n\u001b[0;32m    631\u001b[0m             \u001b[0mparams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    632\u001b[0m         \u001b[0mparams\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_id\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 633\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    634\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    635\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mID\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Python39\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    319\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 321\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    322\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    323\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32mD:\\Python39\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mElementNotInteractableException\u001b[0m: Message: element not interactable\n  (Session info: chrome=94.0.4606.71)\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "source": [
    "driver.find_element_by_xpath('//input[@name=\"selectCheckbox\"]').click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "source": [
    "driver.find_element_by_xpath('//a[@id=\"PageNext\"]').click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "source": [
    "import math\r\n",
    "import time\r\n",
    "from random import random\r\n",
    "import requests_html\r\n",
    "element = driver.find_element_by_xpath('//html')\r\n",
    "main_content =element.get_attribute('outerHTML')  \r\n",
    "html = requests_html.HTML(html= main_content, url='https://localhost/')\r\n",
    "文章数量 = html.xpath('//*[@id=\"countPageDiv\"]/span[1]/em')[0].lxml.text_content()\r\n",
    "print(\"文章数量总计:\",文章数量)\r\n",
    "结果页数 = html.xpath('//*[@id=\"gridTable\"]/div[2]/span[1]')[0].lxml.text_content()[1:4]\r\n",
    "print(\"结果页数总计:\",结果页数)\r\n",
    "pagenum = int(结果页数)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "文章数量总计: 20,722\n",
      "结果页数总计: 120\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "source": [
    "已选文献 = html.xpath('//*[@id=\"selectCount\"]')[0].lxml.text_content()\n",
    "print(int(已选文献))"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "50\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 翻页"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "for page in range(1,pagenum+1):\n",
    "    element = driver.find_element_by_xpath('//html')\n",
    "    main_content =element.get_attribute('outerHTML')  \n",
    "    html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "    已选文献 = html.xpath('//*[@id=\"selectCount\"]')[0].lxml.text_content()\n",
    "    print(int(已选文献))\n",
    "    # 检查文章是否超过500\n",
    "    if int(已选文献) < 500:\n",
    "        # 全选\n",
    "        time.sleep(1+random()*10)\n",
    "        driver.find_element_by_xpath('//input[@name=\"selectCheckbox\"]').click()\n",
    "        # 如果是最后一页，就停止，否则就翻页\n",
    "        if page == int(pagenum):\n",
    "            break\n",
    "        else:\n",
    "            # 翻页\n",
    "            a = driver.find_element_by_xpath('//*[@id=\"PageNext\"]')\n",
    "            a.click()\n",
    "    else:\n",
    "        print(\"已选择500页\")"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "50\n",
      "100\n",
      "150\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 导出refworks文档"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "source": [
    "#打开新窗口\n",
    "driver.switch_to_window(driver.window_handles[1])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "<ipython-input-31-6c6d5ce6602d>:1: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "metadata": {
    "scrolled": true
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [
    "# 注意，在这里要使用鼠标悬停点开隐藏在框里的按钮，不然无法下载"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "source": [
    "move = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]')\n",
    "ActionChains(driver).move_to_element(move).perform()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "source": [
    "move = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]')\n",
    "ActionChains(driver).move_to_element(move).perform()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "source": [
    "move=driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]')\n",
    "ActionChains(driver).move_to_element(move).perform()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "source": [
    "move.click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "source": [
    "driver.switch_to_window(driver.window_handles[2])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "<ipython-input-36-0dfe2a1ada8f>:1: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[2])\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "source": [
    "driver.find_element_by_xpath('/html/body/div[3]/div/div[1]/ul/li[8]').click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "source": [
    "driver.find_element_by_xpath('//*[@id=\"litotxt\"]').click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 导出文件"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "source": [
    "driver.find_element_by_xpath('//*[@id=\"bulkDownload\"]').click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "source": [
    "driver.switch_to_window(driver.window_handles[3])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "<ipython-input-40-1ad0e03448ed>:1: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[3])\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "source": [
    "driver.find_element_by_xpath('//button[@id=\"btn-download-all\"]').click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 封装"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "source": [
    "driver.switch_to_window(driver.window_handles[1])"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "<ipython-input-94-6c6d5ce6602d>:1: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "source": [
    "for page in range(1,pagenum+1):\n",
    "    element = driver.find_element_by_xpath('//html')\n",
    "    main_content =element.get_attribute('outerHTML')  \n",
    "    html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "    已选文献 = html.xpath('//*[@id=\"selectCount\"]')[0].lxml.text_content()\n",
    "    print(int(已选文献))\n",
    "    # Alert Text: 不能超过500个,若要重新选择,请按清除按钮,再进行选取操作\n",
    "    if int(已选文献) < 500:\n",
    "        # 勾选本页文献\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/label').click()\n",
    "        # 翻页\n",
    "        element = driver.find_element_by_xpath('//html')\n",
    "        main_content =element.get_attribute('outerHTML')  \n",
    "        main_content\n",
    "        a = driver.find_element_by_xpath('//*[@id=\"PageNext\"]')\n",
    "        a.click()\n",
    "        # 等待时间 平均时间38*103\n",
    "        time.sleep(3+random()*8)\n",
    "    else:\n",
    "        print(\"已经全选所有页面文献\")\n",
    "        element = driver.find_element_by_xpath('//html')\n",
    "        main_content =element.get_attribute('outerHTML') \n",
    "        main_content\n",
    "        move1 = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]')\n",
    "        ActionChains(driver).move_to_element(move1).perform()\n",
    "        time.sleep(3+random()*8)\n",
    "        move2=driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]')\n",
    "        ActionChains(driver).move_to_element(move2).perform()\n",
    "        time.sleep(3+random()*8)\n",
    "        move3=driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]')\n",
    "        ActionChains(driver).move_to_element(move3).perform()\n",
    "        move3.click()\n",
    "# ----------------------------进入导出页面，导出refworks文档和原文------------------------------------\n",
    "        driver.switch_to_window(driver.window_handles[2])\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element_by_xpath('/html/body/div[3]/div/div[1]/ul/li[8]').click()\n",
    "        driver.find_element_by_xpath('//*[@id=\"litotxt\"]').click()\n",
    "        driver.find_element_by_xpath('//*[@id=\"bulkDownload\"]').click()\n",
    "        driver.switch_to_window(driver.window_handles[3])\n",
    "        driver.find_element_by_xpath('//button[@id=\"btn-download-all\"]').click()\n",
    "# ----------------------------清除已选文章，继续翻页------------------------------------\n",
    "        driver.switch_to.window(driver.window_handles[1])\n",
    "        element = driver.find_element_by_xpath('//html')\n",
    "        main_content =element.get_attribute('outerHTML') \n",
    "        a = driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/a').click()\n",
    "        time.sleep(3+random()*10)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "50\n",
      "100\n",
      "150\n",
      "200\n",
      "250\n",
      "300\n",
      "350\n",
      "400\n",
      "450\n",
      "500\n",
      "已经全选所有页面文献\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "<ipython-input-95-620951d995e8>:35: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[2])\n",
      "<ipython-input-95-620951d995e8>:40: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[3])\n"
     ]
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0\n",
      "50\n",
      "100\n",
      "150\n",
      "200\n",
      "250\n",
      "300\n",
      "350\n",
      "400\n",
      "450\n",
      "500\n",
      "已经全选所有页面文献\n",
      "0\n",
      "50\n",
      "100\n",
      "150\n",
      "200\n",
      "250\n",
      "300\n",
      "350\n",
      "400\n",
      "450\n",
      "500\n",
      "已经全选所有页面文献\n",
      "0\n",
      "50\n",
      "100\n",
      "150\n",
      "200\n",
      "250\n",
      "300\n",
      "350\n",
      "400\n",
      "450\n",
      "500\n",
      "已经全选所有页面文献\n",
      "0\n",
      "50\n",
      "100\n",
      "150\n",
      "200\n",
      "250\n",
      "300\n",
      "350\n",
      "400\n",
      "450\n",
      "500\n",
      "已经全选所有页面文献\n",
      "0\n",
      "50\n",
      "100\n",
      "150\n",
      "200\n",
      "250\n",
      "300\n",
      "350\n",
      "400\n",
      "450\n"
     ]
    },
    {
     "output_type": "error",
     "ename": "IndexError",
     "evalue": "list index out of range",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-95-620951d995e8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m     \u001b[0mmain_content\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0melement\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_attribute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'outerHTML'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mhtml\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrequests_html\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mHTML\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhtml\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0mmain_content\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'https://localhost/'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[0m已选文献\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhtml\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mxpath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'//*[@id=\"selectCount\"]'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlxml\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext_content\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m已选文献\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[1;31m# Alert Text: 不能超过500个,若要重新选择,请按清除按钮,再进行选取操作\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 遇到验证码，需要验证"
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 截图，识别验证码"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "source": [
    "#使用截图，截下验证码，存入F盘中\n",
    "element = driver.find_element_by_id(\"changeVercode\")\n",
    "element.screenshot('F:\\screenImg.png')"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "metadata": {},
     "execution_count": 185
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "source": [
    "#调用讯飞API的印刷文字识别功能，对验证码进行识别\n",
    "#url链接\n",
    "URL = \"http://webapi.xfyun.cn/v1/service/v1/ocr/general\""
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "source": [
    "#API的账户名和key值\n",
    "APPID = \"3c822ee7\"\n",
    "API_KEY = \"ae1dbd7fe29c714277f51eb9cb684313\""
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "source": [
    "#获得头部信息\n",
    "def getHeader():\n",
    "    curTime = str(int(time.time()))\n",
    "    param = {\"language\": \"cn|en\", \"location\": \"false\"}\n",
    "    param = json.dumps(param)\n",
    "    paramBase64 = base64.b64encode(param.encode('utf-8'))\n",
    "    m2 = hashlib.md5()\n",
    "    str1 = API_KEY + curTime + str(paramBase64,'utf-8')\n",
    "    m2.update(str1.encode('utf-8'))\n",
    "    checkSum = m2.hexdigest()\n",
    "    header = {\n",
    "        'X-CurTime': curTime,\n",
    "        'X-Param': paramBase64,\n",
    "        'X-Appid': APPID,\n",
    "        'X-CheckSum': checkSum,\n",
    "        'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',\n",
    "    }\n",
    "    return header\n",
    "        "
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "source": [
    "#存图\n",
    "with open(r\"F:\\screenImg.png\", 'rb') as f:\n",
    "    f1 = f.read()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "source": [
    "#64进制编码图片\n",
    "f1_base64 = str(base64.b64encode(f1), 'utf-8')"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "source": [
    "#输入传送数据\n",
    "data = {\n",
    "        'image': f1_base64\n",
    "        }"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "source": [
    "#得出结果\n",
    "r = requests.post(URL, data=data, headers=getHeader())\n",
    "result = str(r.content, 'utf-8')"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "source": [
    "print(result)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "{\"code\":\"0\",\"data\":{\"block\":[{\"type\":\"text\",\"line\":[{\"confidence\":1,\"word\":[{\"content\":\"QDPL\"}]}]}]},\"desc\":\"success\",\"sid\":\"wcr005f5f9e@gz77e0142e8d0e460b00\"}\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "source": [
    "#从结果中取出验证码\n",
    "yzm = r.json()['data']['block'][0]['line'][0]['word'][0]['content']\n",
    "print(yzm)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "QDPL\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "source": [
    "#先请空验证码框，再将验证码输入其中\n",
    "element=driver.find_element_by_id(\"vericode\")\n",
    "element.clear()\n",
    "element.send_keys(yzm)\n",
    "driver.find_element_by_xpath('//*[@id=\"checkCodeBtn\"]').click()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 输入验证码后，再开始抓取数据"
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "# 重新封装"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "source": [
    "for page in range(1,pagenum+1):\n",
    "    element = driver.find_element_by_xpath('//html')\n",
    "    main_content =element.get_attribute('outerHTML')  \n",
    "    html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "    # Alert Text: 不能超过500个,若要重新选择,请按清除按钮,再进行选取操作\n",
    "    data_list = html.xpath('//*[@id=\"countPageDiv\"]/span[1]/em')\n",
    "    if data_list !=[]:\n",
    "        已选文献 = html.xpath('//*[@id=\"selectCount\"]')[0].lxml.text_content()\n",
    "        print(int(已选文献))\n",
    "        if int(已选文献) < 500:\n",
    "            # 勾选本页文献\n",
    "            time.sleep(3+random()*8)\n",
    "            driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/label').click()\n",
    "            # 翻页\n",
    "            element = driver.find_element_by_xpath('//html')\n",
    "            main_content =element.get_attribute('outerHTML')  \n",
    "            main_content\n",
    "            a = driver.find_element_by_xpath('//*[@id=\"PageNext\"]')\n",
    "            a.click()\n",
    "            # 等待时间 平均时间38*103\n",
    "            time.sleep(3+random()*8)\n",
    "        else:\n",
    "            print(\"已经全选所有页面文献\")\n",
    "            element = driver.find_element_by_xpath('//html')\n",
    "            main_content =element.get_attribute('outerHTML') \n",
    "            main_content\n",
    "            move1 = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]')\n",
    "            ActionChains(driver).move_to_element(move1).perform()\n",
    "            time.sleep(3+random()*8)\n",
    "            move2=driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]')\n",
    "            ActionChains(driver).move_to_element(move2).perform()\n",
    "            time.sleep(3+random()*8)\n",
    "            move3=driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]')\n",
    "            ActionChains(driver).move_to_element(move3).perform()\n",
    "            move3.click()\n",
    "    # ----------------------------进入导出页面---------------------------------------------\n",
    "            driver.switch_to_window(driver.window_handles[2])\n",
    "            time.sleep(3+random()*8)\n",
    "            driver.find_element_by_xpath('/html/body/div[3]/div/div[1]/ul/li[8]').click()\n",
    "            driver.find_element_by_xpath('//*[@id=\"litotxt\"]').click()\n",
    "            driver.find_element_by_xpath('//*[@id=\"bulkDownload\"]').click()\n",
    "            driver.switch_to_window(driver.window_handles[3])\n",
    "            driver.find_element_by_xpath('//button[@id=\"btn-download-all\"]').click()\n",
    "    # ----------------------------清除已选文章，继续翻页-----------------------------------\n",
    "            driver.switch_to.window(driver.window_handles[1])\n",
    "            element = driver.find_element_by_xpath('//html')\n",
    "            main_content =element.get_attribute('outerHTML') \n",
    "            a = driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/a').click()\n",
    "            time.sleep(3+random()*10)\n",
    "    else:\n",
    "        print('需要识别验证码')\n",
    "        element=driver.find_element_by_xpath('//*[@id=\"changeVercode\"]')\n",
    "        element.screenshot('F:\\screenImg.png')\n",
    "        time.sleep(3+random()*10)\n",
    "        \n",
    "        def getHeader():\n",
    "            curTime = str(int(time.time()))\n",
    "            param = {\"language\": \"cn|en\", \"location\": \"false\"}\n",
    "            param = json.dumps(param)\n",
    "            paramBase64 = base64.b64encode(param.encode('utf-8'))\n",
    "            m2 = hashlib.md5()\n",
    "            str1 = API_KEY + curTime + str(paramBase64,'utf-8')\n",
    "            m2.update(str1.encode('utf-8'))\n",
    "            checkSum = m2.hexdigest()\n",
    "            header = {\n",
    "                'X-CurTime': curTime,\n",
    "                'X-Param': paramBase64,\n",
    "                'X-Appid': APPID,\n",
    "                'X-CheckSum': checkSum,\n",
    "                'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',\n",
    "            }\n",
    "            return header\n",
    "            \n",
    "            \n",
    "        with open(r'F:\\screenImg.png', 'rb') as f:\n",
    "            f1 = f.read()\n",
    "        f1_base64 = str(base64.b64encode(f1), 'utf-8')\n",
    "        time.sleep(3+random()*10)\n",
    "        \n",
    "        data = {\n",
    "        'image':f1_base64\n",
    "        }\n",
    "        r = requests.post(URL, data=data, headers=getHeader())\n",
    "        result = str(r.content, 'utf-8')\n",
    "        time.sleep(5)\n",
    "        yzm = r.json()['data']['block'][0]['line'][0]['word'][0]['content']\n",
    "    # ---------------------------将验证码填入输入框并提交----------------------------------\n",
    "        driver.find_element_by_xpath('//*[@id=\"vericode\"]').clear()\n",
    "        driver.find_element_by_xpath('//*[@id=\"vericode\"]').send_keys(yzm)\n",
    "        driver.find_element_by_xpath('//*[@id=\"checkCodeBtn\"]').click()"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "400\n"
     ]
    },
    {
     "output_type": "error",
     "ename": "NoSuchElementException",
     "evalue": "Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"//*[@id=\"PageNext\"]\"}\n  (Session info: chrome=90.0.4430.212)\n",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNoSuchElementException\u001b[0m                    Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-225-66d61f2b96cf>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     16\u001b[0m             \u001b[0mmain_content\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0melement\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_attribute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'outerHTML'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m             \u001b[0mmain_content\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 18\u001b[1;33m             \u001b[0ma\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_xpath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'//*[@id=\"PageNext\"]'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     19\u001b[0m             \u001b[0ma\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclick\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m             \u001b[1;31m# 等待时间 平均时间38*103\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element_by_xpath\u001b[1;34m(self, xpath)\u001b[0m\n\u001b[0;32m    392\u001b[0m             \u001b[0melement\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element_by_xpath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'//div/td[1]'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    393\u001b[0m         \"\"\"\n\u001b[1;32m--> 394\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mXPATH\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mxpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    395\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    396\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfind_elements_by_xpath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mfind_element\u001b[1;34m(self, by, value)\u001b[0m\n\u001b[0;32m    974\u001b[0m                 \u001b[0mby\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCSS_SELECTOR\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    975\u001b[0m                 \u001b[0mvalue\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'[name=\"%s\"]'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 976\u001b[1;33m         return self.execute(Command.FIND_ELEMENT, {\n\u001b[0m\u001b[0;32m    977\u001b[0m             \u001b[1;34m'using'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mby\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    978\u001b[0m             'value': value})['value']\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    319\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 321\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    322\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    323\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNoSuchElementException\u001b[0m: Message: no such element: Unable to locate element: {\"method\":\"xpath\",\"selector\":\"//*[@id=\"PageNext\"]\"}\n  (Session info: chrome=90.0.4430.212)\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.4 64-bit"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "language_info": {
   "name": "python",
   "version": "3.9.4",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "interpreter": {
   "hash": "bd4c67ef142469da7dc4d338a32ac40116904d26076b8e6aa587d80720bc6a2b"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}